package com.study.spark.scala.dataset

import org.apache.spark.sql.SparkSession

/**
  * DataSet操作
  *
  * @author stephen
  * @create 2019-02-26 09:59
  * @since 1.0.0
  */
object DataSetDemo {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName("DataSetDemo")
      .master("local[2]")
      .getOrCreate()
    // 隐式转换
    import spark.implicits._

    // 解析csv文件
    //val df = spark.read.option("header","true").option("inferSchema","true").csv("")
    // 解析json文件
    val df = spark.read.json("")
    df.show()

    // DataFrame转换成DataSet
    val ds = df.as[Info]

    // DataSet和DataFrame基本操作对比
    ds.map(line=>line.name).show()
    df.select("name").show()

    spark.stop()

  }

  case class Info(id: Int, name: String, age: Int)
}
